package com.spider.ask120.webmagic;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;

import javax.management.JMException;

import org.apache.log4j.Logger;
import org.jdiy.core.Args;
import org.jdiy.core.Dao;
import org.jdiy.core.Rs;
import com.alibaba.fastjson.JSONObject;
import com.spider.utils.HttpClientUtil;
import com.spider.utils.JdiyUtils;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.monitor.SpiderMonitor;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康百科频道
 * @author Shan
 *
 */
public class Baike implements PageProcessor{
	public static Logger log = Logger.getLogger(Baike.class);
	static String jkbkUrl = "http://m.120ask.com/baike/";
	static String viewRegex = "http://m.120ask.com/baike/art/[0-9]+";
	private String arti = "http://m.120ask.com/baike/art/";
	static String listUrl = "/baike/\\w+";
	public static final String UTF_8 = "UTF-8";
	public static final String GBK = "GBK";
	static int pageIndex = 1;
	//判断是否是第一次进入
	static int flag = 1;
	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);

	public Site getSite() { 
		return site;
	}

	// process是定制爬虫逻辑的核心接口
	/* (non-Javadoc)
	 * @see us.codecraft.webmagic.processor.PageProcessor#process(us.codecraft.webmagic.Page)
	 */
	public void process(Page page) {
		//进入列表页
		if(page.getUrl().regex(jkbkUrl+"$").match()) { 
			//获取健康百科分类
			List<String> baike = page.getHtml().links().regex(listUrl).all();
			// 去重
			HashSet<String> set = new HashSet<String>(baike);
			baike = new ArrayList<String>();
			set.remove("/baike/art");
			set.remove("/baike/yiyuan");
			baike.addAll(set);
			page.addTargetRequests(baike);

		}else if(page.getUrl().regex(listUrl+"$").match()){
			try {
				Dao dao = JdiyUtils.getDao();

				boolean flag = true;
				while (flag) {
					//获取到话题页列表 
					Map<String, String> params = new HashMap<String, String>();
					params.put("p", pageIndex + "");
					String listJson = HttpClientUtil.getInstance().sendHttpPost(page.getUrl().toString(), params);
					String[] strings = getStrings(listJson);
					if (null != strings) {
						for (String string : strings) {
							String news = unicodeToUtf8(string)+"}";
							int i = news.indexOf("\"i\":\"");
							int t = news.indexOf("\",\"t");
							String id = news.substring(i+5, t);
							String url = page.getUrl().toString();
							String catalog = url.substring(url.indexOf("baike/")+6, url.length());
							//System.out.println("classification:"+catalog);//分类
							//System.out.println("id:"+id);//id
							String title = news.substring(news.indexOf("t\":\"")+4, news.indexOf("\",\"c"));
							//System.out.println("title:" + title);//文章标题
							String abst = news.substring(news.indexOf("s\":\"")+4, news.indexOf("\"}"));
							//System.out.println("abstract:"+abst);//文章简介
							String pic = news.substring(news.indexOf("c\":\"")+4,news.indexOf("\",\"s"));
							//System.out.println("image:"+pic);//文章图片
							//System.out.println(arti+id);
							//						page.addTargetRequest(art + id);//访问id对应详情页

							//存储频道
							Rs ch = dao.rs(new Args("channel","str_id='"+catalog+"' AND parent_id=1"));
							Integer ch_id = null;
							if (ch.isNull()) {//判断频道不存在则添加频道
								Rs channel = new Rs("channel");
								channel.set("str_id", catalog);
								channel.set("parent_id", 1);
								channel.set("level", 2);
								dao.save(channel);
							}else {
								ch_id = ch.getInt("id");
							}
							Rs art = dao.rs(new Args("article","id="+id));
							if (art.isNull() && null != ch_id) {
								//存储文章简介信息
								Rs article = new Rs("article");
								article.set("id", id);
								article.set("channel_id", ch_id);
								article.set("title", title);
								article.set("pic", pic);
								article.set("abst", abst);
								article.set("create_time", new Date());
								dao.save(article);
							}
							//进入到文章正文
							page.addTargetRequest(arti + id);
						}
						pageIndex++;
					}else{
						flag = false;
					}
					break;
				}
			} catch (Exception e) {
				log.info(e);
			}
		}else if(page.getUrl().regex(viewRegex+"$").match()){
			try {
				Dao dao = JdiyUtils.getDao();
				String title = page.getHtml().xpath("//div[@class='question']/h1/text()").get();
				//page.putField("title", title);
				List<String> list = page.getHtml().xpath("//div[@class='questionList clears']/ul/li/a/text()").all();
				//page.putField("contents", list);//目录列表
				//			//page.putField("pageContent", page.getHtml().xpath("//div[@class='answer']/div/html()").all());
				String pic = page.getHtml().xpath("//center/img/@src").get();
				//System.out.println("pic: "+pic);
				int i = 3;
				List<Map<String,Object>> cont = new ArrayList<Map<String,Object>>();
				while (true) {
					String subtitle = page.getHtml().xpath("//div[@class='answer']["+i+"]/h4/text()").get();//小标题
					if (null ==subtitle) {
						break;
					}
					//System.out.println("subtitle: "+(i - 2)+"."+subtitle);
					String content = page.getHtml().xpath("//div[@class='answer']["+i+"]/div/html()").get();//内容
					String con = content.replace("<p class=\"ask_p\"></p>", "");
					//System.out.println("content: "+con);
					Map<String,Object> map = new HashMap<String,Object>();
					map.put((i - 2)+"."+subtitle, con);
					cont.add(map);
					i++;
				}
				String url = page.getUrl().get();
				//文章id
				String id = url.substring(url.indexOf("art/")+4, url.length());
				Rs art = dao.rs(new Args("article","id="+id));
				if (art.isNull()) {
					System.err.println("article is not exist");
				}else {
					//数据存入服务器
					Map<String,Object> map = new HashMap<String, Object>();
					map.put("pic", pic);
					map.put("list", list);
					map.put("cont", cont);
					art.set("content", JSONObject.toJSONString(map));
					dao.save(art);
				}
			} catch (Exception e) {
				e.printStackTrace();
			}

		}

	}
	/**
	 * 根据字符串获取字符串数组
	 * @param str
	 * @return
	 */
	public String[] getStrings(String str){
		String [] strings = null;
		if(null != str && !str.isEmpty()){
			String string = str.substring(2, str.length()-1);
			strings = string.split("},");
		}
		return strings;
	}
	/**
	 * unicode转utf8
	 * @param theString
	 * @return
	 */
	public static String unicodeToUtf8(String theString) {
		if (null == theString || theString.isEmpty()) {
			return null;
		}
		char aChar;
		int len = theString.length();
		StringBuffer outBuffer = new StringBuffer(len);
		for (int x = 0; x < len;) {
			aChar = theString.charAt(x++);
			if (aChar == '\\') {
				aChar = theString.charAt(x++);
				if (aChar == 'u') {
					// Read the xxxx
					int value = 0;
					for (int i = 0; i < 4; i++) {
						aChar = theString.charAt(x++);
						switch (aChar) {
						case '0':
						case '1':
						case '2':
						case '3':
						case '4':
						case '5':
						case '6':
						case '7':
						case '8':
						case '9':
							value = (value << 4) + aChar - '0';
							break;
						case 'a':
						case 'b':
						case 'c':
						case 'd':
						case 'e':
						case 'f':
							value = (value << 4) + 10 + aChar - 'a';
							break;
						case 'A':
						case 'B':
						case 'C':
						case 'D':
						case 'E':
						case 'F':
							value = (value << 4) + 10 + aChar - 'A';
							break;
						default:
							throw new IllegalArgumentException(
									"Malformed   \\uxxxx   encoding.");
						}
					}
					outBuffer.append((char) value);
				} else {
					if (aChar == 't')
						aChar = '\t';
					else if (aChar == 'r')
						aChar = '\r';
					else if (aChar == 'n')
						aChar = '\n';
					else if (aChar == 'f')
						aChar = '\f';
					outBuffer.append(aChar);
				}
			} else
				outBuffer.append(aChar);
		}
		return outBuffer.toString();
	}

	public static void main(String[] args) throws JMException {
		Spider s = Spider.create(new Baike()).addUrl("http://m.120ask.com/baike/");
		SpiderMonitor.instance().register(s);
		s.thread(1).start();
	}
}
